{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "distribute strategy",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D5M-9Lej0Szq",
        "colab_type": "text"
      },
      "source": [
        "install 2.0 with gpu support\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BBoJ9dBEz7Tu",
        "colab_type": "code",
        "outputId": "85cd6405-e926-4187-bd7f-88f97fe229de",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 615
        }
      },
      "source": [
        "pip install tf-nightly-gpu"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tf-nightly-gpu in /usr/local/lib/python3.6/dist-packages (2.2.0.dev20200114)\n",
            "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (0.33.6)\n",
            "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (2.10.0)\n",
            "Requirement already satisfied: scipy==1.4.1; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.4.1)\n",
            "Requirement already satisfied: gast==0.3.2 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (0.3.2)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.15.0)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (3.10.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (0.8.1)\n",
            "Requirement already satisfied: tf-estimator-nightly in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (2.1.0.dev2020011409)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.1.0)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (0.9.0)\n",
            "Requirement already satisfied: tb-nightly<2.3.0a0,>=2.2.0a0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (2.2.0a20200106)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (3.1.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.1.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.11.2)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.12.0)\n",
            "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (1.17.5)\n",
            "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tf-nightly-gpu) (0.1.8)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->tf-nightly-gpu) (42.0.2)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (2.21.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (3.1.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (0.16.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (1.10.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (0.4.1)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (3.0.4)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (2.8)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (2019.11.28)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (0.2.7)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (4.0.0)\n",
            "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (4.0)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (1.3.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tb-nightly<2.3.0a0,>=2.2.0a0->tf-nightly-gpu) (3.1.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TFGm0TXM0bak",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "import os\n",
        "import time\n",
        "import tensorflow as tf\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xvPuXIyXAd5G",
        "colab_type": "code",
        "outputId": "b926b9ee-62e4-4d10-90e4-21f32f5ace6f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tf.test.gpu_device_name()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/device:GPU:0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wYar3A0vBVtZ",
        "colab_type": "code",
        "outputId": "0f00edf9-6339-472e-9983-f09bfb3bc00b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from tensorflow.python.client import device_lib\n",
        "\n",
        "def get_available_gpus():\n",
        "    local_device_protos = device_lib.list_local_devices()\n",
        "    return [x.name for x in local_device_protos if x.device_type == 'GPU']\n",
        "\n",
        "get_available_gpus()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['/device:GPU:0']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZiyzTwJ20iaY",
        "colab_type": "text"
      },
      "source": [
        "train dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K_4FCxsO0les",
        "colab_type": "code",
        "outputId": "800c2e6b-f9af-4d85-bf5a-b2796f25effb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# How many categories we are predicting from (0-9)\n",
        "LABEL_DIMENSIONS = 10\n",
        "\n",
        "\n",
        "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.fashion_mnist.load_data()\n",
        "TRAINING_SIZE = len(train_images)\n",
        "TEST_SIZE = len(test_images)\n",
        "\n",
        "train_images = np.asarray(train_images, dtype=np.float32) / 255\n",
        "\n",
        "# Convert the train images and add channels\n",
        "train_images = train_images.reshape((TRAINING_SIZE, 28, 28, 1))\n",
        "\n",
        "test_images = np.asarray(test_images, dtype=np.float32) / 255\n",
        "# Convert the train images and add channels\n",
        "test_images = test_images.reshape((TEST_SIZE, 28, 28, 1))\n",
        "\n",
        "train_labels  = tf.keras.utils.to_categorical(train_labels, LABEL_DIMENSIONS)\n",
        "test_labels = tf.keras.utils.to_categorical(test_labels, LABEL_DIMENSIONS)\n",
        "\n",
        "# Cast the labels to float\n",
        "train_labels = train_labels.astype(np.float32)\n",
        "test_labels = test_labels.astype(np.float32)\n",
        "print (train_labels.shape)\n",
        "print (test_labels.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(60000, 10)\n",
            "(10000, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_eV9AWWb08RP",
        "colab_type": "text"
      },
      "source": [
        "build a CNN model - using the functional API"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9z5ScWig0-XR",
        "colab_type": "code",
        "outputId": "21cabae1-31af-45d4-b9f6-79c3e917dce4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 527
        }
      },
      "source": [
        "inputs = tf.keras.Input(shape=(28,28,1))  \n",
        "x = tf.keras.layers.Conv2D(filters=32, kernel_size=(3, 3), activation='relu')(inputs)\n",
        "x = tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=2)(x)\n",
        "x = tf.keras.layers.Conv2D(filters=64, kernel_size=(3, 3), activation='relu')(x)\n",
        "x = tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=2)(x)\n",
        "x = tf.keras.layers.Conv2D(filters=256, kernel_size=(3, 3), activation='relu')(x)\n",
        "x = tf.keras.layers.Flatten()(x)\n",
        "x = tf.keras.layers.Dense(64, activation='relu')(x)\n",
        "x = tf.keras.layers.Dropout(0.2)(x)\n",
        "x = tf.keras.layers.Dense(128, activation='relu')(x)\n",
        "\n",
        "predictions = tf.keras.layers.Dense(LABEL_DIMENSIONS, activation='softmax')(x)\n",
        "\n",
        "model = tf.keras.Model(inputs=inputs, outputs=predictions)\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         [(None, 28, 28, 1)]       0         \n",
            "_________________________________________________________________\n",
            "conv2d (Conv2D)              (None, 26, 26, 32)        320       \n",
            "_________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D) (None, 13, 13, 32)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 11, 11, 64)        18496     \n",
            "_________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2 (None, 5, 5, 64)          0         \n",
            "_________________________________________________________________\n",
            "conv2d_2 (Conv2D)            (None, 3, 3, 256)         147712    \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 2304)              0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 64)                147520    \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 128)               8320      \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 10)                1290      \n",
            "=================================================================\n",
            "Total params: 323,658\n",
            "Trainable params: 323,658\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M9hlsler2JuV",
        "colab_type": "text"
      },
      "source": [
        "compile the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bm1_OUjK1374",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = tf.keras.optimizers.SGD()\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer=optimizer,\n",
        "              metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rHuut1Li2O8b",
        "colab_type": "text"
      },
      "source": [
        "define the strategy\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FYZm9kgI2SMa",
        "colab_type": "code",
        "outputId": "48449604-ec10-49b8-f0b0-029b47a22b45",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "#strategy = None\n",
        "strategy = tf.distribute.MirroredStrategy()\n",
        "config = tf.estimator.RunConfig(train_distribute=strategy)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0',)\n",
            "INFO:tensorflow:Initializing RunConfig with distribution strategies.\n",
            "INFO:tensorflow:Not using Distribute Coordinator.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P-3Ynj6R4wGq",
        "colab_type": "text"
      },
      "source": [
        "convert Keras to Estimator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eVP6xiGY43Sn",
        "colab_type": "code",
        "outputId": "cb2534c8-de7f-42be-833c-b728d5f9b3d2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        }
      },
      "source": [
        "estimator = tf.keras.estimator.model_to_estimator(model, config=config)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpvmtauxwy\n",
            "INFO:tensorflow:Using the Keras model provided.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1643: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "If using Keras pass *_constraint arguments to layers.\n",
            "INFO:tensorflow:Using config: {'_model_dir': '/tmp/tmpvmtauxwy', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
            "graph_options {\n",
            "  rewrite_options {\n",
            "    meta_optimizer_iterations: ONE\n",
            "  }\n",
            "}\n",
            ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': <tensorflow.python.distribute.mirrored_strategy.MirroredStrategy object at 0x7fe6cc73f518>, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': ClusterSpec({}), '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_distribute_coordinator_mode': None}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PlsFvTOu5Axk",
        "colab_type": "text"
      },
      "source": [
        "input function for estimator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uQq0R8cc485C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def input_fn(images, labels, epochs, batch_size):\n",
        "    # Convert the inputs to a Dataset\n",
        "    dataset = tf.data.Dataset.from_tensor_slices((images, labels))\n",
        "\n",
        "    # Shuffle, repeat, and batch the examples. \n",
        "    SHUFFLE_SIZE = 5000\n",
        "    dataset = dataset.shuffle(SHUFFLE_SIZE).repeat(epochs).batch(batch_size)\n",
        "    dataset = dataset.prefetch(None)\n",
        "\n",
        "    # Return the dataset. \n",
        "    return dataset"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kt25OEKS5g75",
        "colab_type": "text"
      },
      "source": [
        "train function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MMbPOC3f5ku3",
        "colab_type": "code",
        "outputId": "b2f5e783-5b22-4631-a22f-25318ff05df7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "BATCH_SIZE = 512\n",
        "EPOCHS = 50\n",
        "\n",
        "#time_hist = TimeHistory()\n",
        "\n",
        "estimator_train_result = estimator.train(input_fn=lambda:input_fn(train_images,\n",
        "                                         train_labels,\n",
        "                                         epochs=EPOCHS,\n",
        "                                         batch_size=BATCH_SIZE))\n",
        "print(estimator_train_result)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Reduce to /replica:0/task:0/device:CPU:0 then broadcast to ('/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Calling model_fn.\n",
            "INFO:tensorflow:Reduce to /replica:0/task:0/device:CPU:0 then broadcast to ('/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /replica:0/task:0/device:CPU:0 then broadcast to ('/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /replica:0/task:0/device:CPU:0 then broadcast to ('/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /replica:0/task:0/device:CPU:0 then broadcast to ('/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /replica:0/task:0/device:CPU:0 then broadcast to ('/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /replica:0/task:0/device:CPU:0 then broadcast to ('/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /replica:0/task:0/device:CPU:0 then broadcast to ('/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /replica:0/task:0/device:CPU:0 then broadcast to ('/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Reduce to /replica:0/task:0/device:CPU:0 then broadcast to ('/replica:0/task:0/device:CPU:0',).\n",
            "INFO:tensorflow:Done calling model_fn.\n",
            "WARNING:tensorflow:AutoGraph could not transform <function _combine_distributed_scaffold.<locals>.<lambda> at 0x7fe6cc709ea0> and will run it as-is.\n",
            "Cause: could not parse the source code:\n",
            "\n",
            "      lambda scaffold: scaffold.ready_op, args=(grouped_scaffold,))\n",
            "\n",
            "This error may be avoided by creating the lambda in a standalone statement.\n",
            "\n",
            "WARNING: AutoGraph could not transform <function _combine_distributed_scaffold.<locals>.<lambda> at 0x7fe6cc709ea0> and will run it as-is.\n",
            "Cause: could not parse the source code:\n",
            "\n",
            "      lambda scaffold: scaffold.ready_op, args=(grouped_scaffold,))\n",
            "\n",
            "This error may be avoided by creating the lambda in a standalone statement.\n",
            "\n",
            "INFO:tensorflow:Warm-starting with WarmStartSettings: WarmStartSettings(ckpt_to_initialize_from='/tmp/tmpvmtauxwy/keras/keras_model.ckpt', vars_to_warm_start='.*', var_name_to_vocab_info={}, var_name_to_prev_var_name={})\n",
            "INFO:tensorflow:Warm-starting from: /tmp/tmpvmtauxwy/keras/keras_model.ckpt\n",
            "INFO:tensorflow:Warm-starting variables only in TRAINABLE_VARIABLES.\n",
            "INFO:tensorflow:Warm-started 12 variables.\n",
            "INFO:tensorflow:Create CheckpointSaverHook.\n",
            "INFO:tensorflow:Graph was finalized.\n",
            "INFO:tensorflow:Running local_init_op.\n",
            "INFO:tensorflow:Done running local_init_op.\n",
            "INFO:tensorflow:Saving checkpoints for 0 into /tmp/tmpvmtauxwy/model.ckpt.\n",
            "INFO:tensorflow:loss = 2.3019295, step = 0\n",
            "INFO:tensorflow:global_step/sec: 120.023\n",
            "INFO:tensorflow:loss = 2.2863631, step = 100 (0.834 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.774\n",
            "INFO:tensorflow:loss = 2.2581525, step = 200 (0.780 sec)\n",
            "INFO:tensorflow:global_step/sec: 129.642\n",
            "INFO:tensorflow:loss = 2.1821587, step = 300 (0.769 sec)\n",
            "INFO:tensorflow:global_step/sec: 124.298\n",
            "INFO:tensorflow:loss = 1.8901887, step = 400 (0.806 sec)\n",
            "INFO:tensorflow:global_step/sec: 125.772\n",
            "INFO:tensorflow:loss = 1.2512764, step = 500 (0.795 sec)\n",
            "INFO:tensorflow:global_step/sec: 126.129\n",
            "INFO:tensorflow:loss = 1.1520438, step = 600 (0.794 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.564\n",
            "INFO:tensorflow:loss = 0.93469757, step = 700 (0.783 sec)\n",
            "INFO:tensorflow:global_step/sec: 130.392\n",
            "INFO:tensorflow:loss = 0.96332175, step = 800 (0.766 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.696\n",
            "INFO:tensorflow:loss = 0.8922721, step = 900 (0.782 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.488\n",
            "INFO:tensorflow:loss = 0.8417127, step = 1000 (0.787 sec)\n",
            "INFO:tensorflow:global_step/sec: 129.403\n",
            "INFO:tensorflow:loss = 0.8108121, step = 1100 (0.773 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.852\n",
            "INFO:tensorflow:loss = 0.8237517, step = 1200 (0.773 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.847\n",
            "INFO:tensorflow:loss = 0.7075269, step = 1300 (0.784 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.969\n",
            "INFO:tensorflow:loss = 0.6900586, step = 1400 (0.786 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.095\n",
            "INFO:tensorflow:loss = 0.7041669, step = 1500 (0.783 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.261\n",
            "INFO:tensorflow:loss = 0.7485477, step = 1600 (0.785 sec)\n",
            "INFO:tensorflow:global_step/sec: 131.291\n",
            "INFO:tensorflow:loss = 0.63827455, step = 1700 (0.762 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.48\n",
            "INFO:tensorflow:loss = 0.72328836, step = 1800 (0.779 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.912\n",
            "INFO:tensorflow:loss = 0.6695744, step = 1900 (0.779 sec)\n",
            "INFO:tensorflow:global_step/sec: 129.993\n",
            "INFO:tensorflow:loss = 0.6532612, step = 2000 (0.772 sec)\n",
            "INFO:tensorflow:global_step/sec: 129.1\n",
            "INFO:tensorflow:loss = 0.7489969, step = 2100 (0.775 sec)\n",
            "INFO:tensorflow:global_step/sec: 126.687\n",
            "INFO:tensorflow:loss = 0.56666976, step = 2200 (0.788 sec)\n",
            "INFO:tensorflow:global_step/sec: 130.035\n",
            "INFO:tensorflow:loss = 0.6296058, step = 2300 (0.770 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.291\n",
            "INFO:tensorflow:loss = 0.7053249, step = 2400 (0.787 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.003\n",
            "INFO:tensorflow:loss = 0.63421464, step = 2500 (0.787 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.961\n",
            "INFO:tensorflow:loss = 0.61893505, step = 2600 (0.776 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.311\n",
            "INFO:tensorflow:loss = 0.5682474, step = 2700 (0.780 sec)\n",
            "INFO:tensorflow:global_step/sec: 125.58\n",
            "INFO:tensorflow:loss = 0.6327102, step = 2800 (0.793 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.082\n",
            "INFO:tensorflow:loss = 0.60169095, step = 2900 (0.783 sec)\n",
            "INFO:tensorflow:global_step/sec: 129.657\n",
            "INFO:tensorflow:loss = 0.61266655, step = 3000 (0.769 sec)\n",
            "INFO:tensorflow:global_step/sec: 129.356\n",
            "INFO:tensorflow:loss = 0.5938358, step = 3100 (0.773 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.206\n",
            "INFO:tensorflow:loss = 0.604183, step = 3200 (0.780 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.993\n",
            "INFO:tensorflow:loss = 0.59179205, step = 3300 (0.779 sec)\n",
            "INFO:tensorflow:global_step/sec: 130.366\n",
            "INFO:tensorflow:loss = 0.56381416, step = 3400 (0.764 sec)\n",
            "INFO:tensorflow:global_step/sec: 126.768\n",
            "INFO:tensorflow:loss = 0.59084624, step = 3500 (0.791 sec)\n",
            "INFO:tensorflow:global_step/sec: 129.788\n",
            "INFO:tensorflow:loss = 0.571569, step = 3600 (0.769 sec)\n",
            "INFO:tensorflow:global_step/sec: 131.234\n",
            "INFO:tensorflow:loss = 0.58073723, step = 3700 (0.765 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.719\n",
            "INFO:tensorflow:loss = 0.5684445, step = 3800 (0.773 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.722\n",
            "INFO:tensorflow:loss = 0.59569657, step = 3900 (0.783 sec)\n",
            "INFO:tensorflow:global_step/sec: 130.747\n",
            "INFO:tensorflow:loss = 0.50342244, step = 4000 (0.765 sec)\n",
            "INFO:tensorflow:global_step/sec: 130.184\n",
            "INFO:tensorflow:loss = 0.5135074, step = 4100 (0.770 sec)\n",
            "INFO:tensorflow:global_step/sec: 129.725\n",
            "INFO:tensorflow:loss = 0.5251267, step = 4200 (0.773 sec)\n",
            "INFO:tensorflow:global_step/sec: 126.098\n",
            "INFO:tensorflow:loss = 0.5489641, step = 4300 (0.790 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.101\n",
            "INFO:tensorflow:loss = 0.5329821, step = 4400 (0.787 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.878\n",
            "INFO:tensorflow:loss = 0.5168418, step = 4500 (0.785 sec)\n",
            "INFO:tensorflow:global_step/sec: 130.997\n",
            "INFO:tensorflow:loss = 0.473872, step = 4600 (0.763 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.565\n",
            "INFO:tensorflow:loss = 0.46939266, step = 4700 (0.785 sec)\n",
            "INFO:tensorflow:global_step/sec: 131.12\n",
            "INFO:tensorflow:loss = 0.51474804, step = 4800 (0.759 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.973\n",
            "INFO:tensorflow:loss = 0.43466482, step = 4900 (0.782 sec)\n",
            "INFO:tensorflow:global_step/sec: 129.114\n",
            "INFO:tensorflow:loss = 0.48647282, step = 5000 (0.778 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.849\n",
            "INFO:tensorflow:loss = 0.4595028, step = 5100 (0.779 sec)\n",
            "INFO:tensorflow:global_step/sec: 127.955\n",
            "INFO:tensorflow:loss = 0.5078115, step = 5200 (0.781 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.909\n",
            "INFO:tensorflow:loss = 0.5008959, step = 5300 (0.780 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.215\n",
            "INFO:tensorflow:loss = 0.4784897, step = 5400 (0.776 sec)\n",
            "INFO:tensorflow:global_step/sec: 128.072\n",
            "INFO:tensorflow:loss = 0.4643142, step = 5500 (0.781 sec)\n",
            "INFO:tensorflow:global_step/sec: 129.382\n",
            "INFO:tensorflow:loss = 0.5441301, step = 5600 (0.776 sec)\n",
            "INFO:tensorflow:global_step/sec: 122.042\n",
            "INFO:tensorflow:loss = 0.49887368, step = 5700 (0.816 sec)\n",
            "INFO:tensorflow:global_step/sec: 125.046\n",
            "INFO:tensorflow:loss = 0.47165042, step = 5800 (0.802 sec)\n",
            "INFO:tensorflow:Saving checkpoints for 5860 into /tmp/tmpvmtauxwy/model.ckpt.\n",
            "INFO:tensorflow:Loss for final step: 0.49285737.\n",
            "<tensorflow_estimator.python.estimator.estimator.EstimatorV2 object at 0x7fe6cc72e208>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JDcD0hKZ9LOK",
        "colab_type": "text"
      },
      "source": [
        "with 2.0alpha If i use strategy = tf.distribute.MirroredStrategy() and GPUS\n",
        "then I get an error \"ValueError: Only TensorFlow native optimizers are supported with DistributionStrategy.\"\n",
        "**Reported** the problem to  https://github.com/tensorflow/tensorflow/issues/27696\n",
        "fixed in night-build"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eWru8nRWvmIu",
        "colab_type": "code",
        "outputId": "d004abec-8aff-4619-fd0c-b31034c99cf8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "estimator.evaluate(lambda:input_fn(test_images, \n",
        "                                   test_labels,\n",
        "                                   epochs=1,\n",
        "                                   batch_size=BATCH_SIZE))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Calling model_fn.\n",
            "INFO:tensorflow:Done calling model_fn.\n",
            "INFO:tensorflow:Starting evaluation at 2020-01-14T19:53:28Z\n",
            "INFO:tensorflow:Graph was finalized.\n",
            "INFO:tensorflow:Restoring parameters from /tmp/tmpvmtauxwy/model.ckpt-5860\n",
            "INFO:tensorflow:Running local_init_op.\n",
            "INFO:tensorflow:Done running local_init_op.\n",
            "INFO:tensorflow:Inference Time : 0.46057s\n",
            "INFO:tensorflow:Finished evaluation at 2020-01-14-19:53:29\n",
            "INFO:tensorflow:Saving dict for global step 5860: accuracy = 0.8224, global_step = 5860, loss = 0.4785419\n",
            "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 5860: /tmp/tmpvmtauxwy/model.ckpt-5860\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'accuracy': 0.8224, 'global_step': 5860, 'loss': 0.4785419}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "37HXKdlUEOnQ",
        "colab_type": "text"
      },
      "source": [
        "In alternative this is also useful"
      ]
    }
  ]
}